# -*- coding:utf-8 -*-
# @Author: shenyuyu
# @Time: 2023/6/28 10:34
# @File: qu_1_spark_sql_dataFrame.py
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StringType, IntegerType

if __name__ == '__main__':
    spark = SparkSession.builder.appName("a").master("local[*]").getOrCreate()
    sc = spark.sparkContext
    rdd1 = sc.textFile("file:///tmp/pycharm_project_161/data/sql/people.txt")
    rdd2 = rdd1.map(lambda x: x.split(",")).map(lambda x: (x[0], int(x[1])))

    # print(rdd2.collect())
    # df = spark.createDataFrame(rdd2, schema=["name", "age"])
    # df = spark.createDataFrame(rdd2, schema="name String, age Int")
    # schema = StructType().add("name", StringType(), False).add("age", IntegerType(), False)
    # df = spark.createDataFrame(rdd2, schema=schema)
    # df.printSchema()
    # df.show()


    schema = StructType() \
        .add("name", StringType(), False) \
        .add("age", IntegerType(), False)
    rdd2.toDF(schema=["name", "age"]).show()
    rdd2.toDF(schema=schema).show()